from WeirLinksCrawler import getLinks
from WeirdCommon import getContext
from crawler import config
import opencc
from file_dir import file_retrieve
import time
from db.NovelInfoDBOperator import NovelInfo, DBOperator
from requests.exceptions import ConnectionError

prefix = "https://1337789-vvm.c11oz688.cc/pw/thread.php?fid=194&page="
link_prefix = "https://1337789-vvm.c11oz688.cc/pw/"
output_path_prefix = config.generalConfig.novel_output_path
# 创建一个转换器，设置从繁体到简体的转换
converter = opencc.OpenCC('t2s')

class WeirdNovel:
    def __init__(self):
        self.name = ''
        self.content = ''
        self.url = ''

    def analyze_content(self, soap):
        novel = WeirdNovel()
        novel.name = converter.convert(soap.find(id='subject_tpc').text)
        print(novel.name)
        tmp_content = soap.find(id='read_tpc')
        for br_tag in tmp_content.find_all('br'):
            br_tag.replace_with(r'\n')
        tmp_content_str = converter.convert(tmp_content.text)
        lines = tmp_content_str.split(r'\n')

        novel.content = lines
        return novel

    def output_novel(self, novel):
        dbsession = DBOperator()
        novel_info = NovelInfo()
        novel_info.title = novel.name
        novel_info.content = novel.content
        novel_info.parent_url = novel.url
        novel_info.crawled_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        novel_info.url = novel.url[novel.url.rfind('/')+1:]
        # if (file_retrieve.file_exists(output_path_prefix, novel.name + '.txt')
        #         and dbsession.check_novel_exist_count(novel_info) > 0):
        if dbsession.check_novel_exist_count(novel_info) > 0:
            print(novel.name + 'is Exist!')
            pass
        with open(output_path_prefix + novel.name + '.txt', mode='a', encoding='utf-8') as file:
            file.write(novel.name)
            file.write('\n\n')
            for line in novel.content:
                file.write(line.strip())
                file.write('\n')
            file.write('\n\n')

            dbsession.add_novel_info(novel_info)


    def novel_download(self):
        links = getLinks(prefix, 1, 100)
        for link in links:
            print(link)
            retry = True
            while retry:
                try:
                    w_novel = WeirdNovel()
                    content = getContext(link_prefix + link)
                    novel = w_novel.analyze_content(content)
                    novel.url = link_prefix + link
                    w_novel.output_novel(novel)
                    retry = False
                except ConnectionError as e:
                    print(r'ConnectionError invoked!')
                    retry = True
        print('Finish!')
        return 1





if __name__ == '__main__':
    w_novel = WeirdNovel()
    w_novel.novel_download()

